﻿using System.Threading;
using DM4;
using sscore;
using System;
using System.Collections.Generic;
using sscore.Utils;
using System.Text.RegularExpressions;
using System.IO;
using sscore.Requests;

namespace CA
{
    public class CA : ModuleBase
    {
        private const string DownloadByCounties = "downloadbycounties";
        private readonly string _searchUrlByCounties;
        public string _searchUrlByNames;
        private readonly string _offenderLink;
        private const string StateName = "CA";
        private ResultFile _imagesResultFile;

        private readonly String[] _counties = new[] { 
            "ALAMEDA","ALPINE","AMADOR","BUTTE","CALAVERAS","COLUSA","CONTRA COSTA","DEL NORTE","EL DORADO","FRESNO",
            "GLENN","HUMBOLDT","IMPERIAL","INYO","KERN","KINGS","LAKE","LASSEN","LOS ANGELES","MADERA",
            "MARIN","MARIPOSA","MENDOCINO","MERCED","MODOC","MONO","MONTEREY","NAPA","NEVADA","ORANGE",
            "PLACER","PLUMAS","RIVERSIDE","SACRAMENTO","SAN BENITO","SAN BERNARDINO","SAN DIEGO","SAN FRANCISCO","SAN JOAQUIN","SAN LUIS OBISPO",
            "SAN MATEO","SANTA BARBARA","SANTA CLARA","SANTA CRUZ","SHASTA","SIERRA","SISKIYOU","SOLANO","SONOMA","STANISLAUS",
            "SUTTER","TEHAMA","TRINITY","TULARE","TUOLUMNE","VENTURA","YOLO","YUBA"
            };
        const string SearchParamFileName = "params.txt";

        public CA()
        {
            //initialize main system constants
            ModuleCode = "SO_CA";
            Description = "CA Sex Offenders";
            Version = "1.0";

            //for creating sessions
            StartParam = 0;
            StopParam = 130250;

            //initialize help 
            AddToHelp(DownloadByCounties, "true|false");
            AddToHelp(DM4Consts.StartPosition, String.Format("{0}...{1}|{2}", StartParam, _counties.Length - 1, StopParam),
                String.Format("{0} value for search by county, {1} value for search by names", _counties.Length - 1, StopParam));
            AddToHelp(DM4Consts.EndPositon, String.Format("{0}...{1}|{2}", StartParam, _counties.Length - 1, StopParam),
                String.Format("{0} value for search by county, {1} value for search by names", _counties.Length - 1, StopParam));

            AddToHelp(DM4Consts.IsDownload, "true|false");
            AddToHelp(DM4Consts.IsDownloadImg, "true|false");
            AddToHelp(DM4Consts.InsertToDb, "false|true", "(Expected: servername, dbname)");
            AddToHelp(DM4Consts.ServerName, "");
            AddToHelp(DM4Consts.DbName, "");
            AddToHelp(DM4Consts.IsClearDb, "false|true");
            AddToHelp(DM4Consts.TestParam, "false|true");

            RootUrl = "http://www.meganslaw.ca.gov/";
            _searchUrlByCounties = String.Format("{0}cgi/prosoma.dll?searchby=CountyList&SelectCounty={{0}}&SB=0&PageNo={{1}}", RootUrl);
            _searchUrlByNames = String.Format("{0}cgi/prosoma.dll?searchby=namelist&lastName={{0}}&SB=0&PageNo={{1}}", RootUrl);

            _offenderLink = String.Format("{0}cgi/prosoma.dll?searchby=offender&id={{0}}", RootUrl);
        }

        private void SetParametersDefaults()
        {
            Params.CreateBooleanParam(DownloadByCounties, true);
            Params.CreateBooleanParam(DM4Consts.IsDownload, true);
            Params.CreateBooleanParam(DM4Consts.IsDownloadImg, true);
            Params.CreateBooleanParam(DM4Consts.InsertToDb, false);
            Params.CreateBooleanParam(DM4Consts.IsClearDb, false);
            Params.CreateStringParam(DM4Consts.DbName, "");
            Params.CreateStringParam(DM4Consts.ServerName, "");
            Params.CreateIntegerParam(DM4Consts.StartPosition, StartParam);
            Params.CreateIntegerParam(DM4Consts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
            Params.CreateIntegerParam(DM4Consts.EndPositon, StopParam);
            Params.CreateIntegerParam(DM4Consts.LastRecord, 0);
            Params.CreateIntegerParam(DmConsts.LastPage, 1); //default value is 1
        }

        public override OperationResult Init(IDM4Core iCore, DmTaskParams param)
        {
            //use basic initialization sequence
            base.Init(iCore, param);

            SetParametersDefaults();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult Start()
        {
            DoWork();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult DoWork()
        {
            OperationResult or = new OperationResult { Result = DM4Consts.ErrorUnknown };

            if (!Ct.IsCancellationRequested)
            {
                WriteLog("Start parameters:", MessageType.Warning);
                foreach (KeyValuePair<string, string> kvp in Params.GetDictionary()) //get all existing parameters
                    WriteLog(String.Format("Key = {0}, Value = {1}", kvp.Key, kvp.Value), MessageType.Warning);

                if (Params.GetBooleanParam(DM4Consts.IsDownload))
                    Load();

                if (Params.GetBooleanParam(DM4Consts.InsertToDb))
                    if ((Params.GetStringParam(DM4Consts.DbName) == "") || (Params.GetStringParam(DM4Consts.ServerName) == ""))
                    {
                        or.Message = String.Format("{0} or {1} is empty", DM4Consts.DbName, DM4Consts.ServerName);
                        ResultCode = DM4Consts.SessionResultCodeError;
                        ResultMessage = or.Message;
                        UpdateStatus(DmConsts.Error);
                        WriteErrorLog(or.Message); //writting to error and session log   

                        return or;
                    }
                    else
                    {
                        //insert offenders to db 
                        UpdateStatus(DM4Consts.SessionInsertToDb);

                        Offenders.OffInsertToDatabase(StateName, Params.GetStringParam(DmConsts.DbName),
                             Params.GetStringParam(DmConsts.ServerName), ResultDir, Params.GetBooleanParam(DmConsts.IsClearDb),
                             WriteLog);

                    }

                //check if need abort task
                if (Ct.IsCancellationRequested)
                {
                    ResultCode = DM4Consts.SessionResultTerminateByUser;
                    ResultMessage = "Session terminated by user";
                    UpdateStatus(DM4Consts.SessionWait);
                    WriteLog("Session aborted by user", MessageType.Cool);
                }
                else
                {
                    UpdateProgress(1);
                    ResultCode = DM4Consts.SessionResultCodeOk;
                    ResultMessage = String.Format("Session complete, errors : {0}, records : {1}", TaskParams.ErrorCount, TaskParams.RecordCount);
                    UpdateStatus(DM4Consts.SessionComplete);
                    WriteLog("------------Finish----------------", MessageType.Cool);
                }
            }
            else
            {
                ResultCode = DM4Consts.SessionResultTerminateByUser;
                ResultMessage = "Session terminated by user";
                UpdateStatus(DM4Consts.SessionWait);
                WriteLog("Session aborted by user", MessageType.Cool);
            }


            return or;
        }

        private void Load()
        {
            string resultFileName = String.Format("{0}\\{1}.{2}", ResultDir, TaskParams.SessionId, DmConsts.ResultFilesExtension);
            bool workFlag = true;

            ResultFile = new ResultFile(resultFileName, OffendersResult.GetHeader());
            _imagesResultFile = new ResultFile(String.Format("{0}\\{1}_images.{2}", ResultDir, TaskParams.SessionId, DM4Consts.ResultFilesExtension), "FileName|FileUrl");
            try
            {
                while (workFlag)
                {
                    //рабочий цикл
                    string error;

                    try
                    {
                        error = "";  //Required!!  /*if (!String.IsNullOrEmpty(error)) --> restart thread; else --> download finished*/

                        WriteLog("Start download...");

                        if (Params.GetBooleanParam(DownloadByCounties))
                            SearchingByCounties();
                        else
                            SearchingByNames();
                    }
                    catch (Exception e)
                    {
                        error = e.Message;
                    }

                    if (!String.IsNullOrEmpty(error))
                    {
                        Thread.Sleep(30000);
                        WriteErrorLog(error);
                        WriteLog("Try continue work...", MessageType.Warning);
                    }
                    else
                    {
                        workFlag = false;  //все нормально откачало, заканчиваем работу
                    }
                }
            }
            finally
            {
                ResultFile.Close();
                _imagesResultFile.Close();
            }
        }

        private void SearchingByCounties()
        {
            if (Test)
            {
                Params.SetIntegerParam(DmConsts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
                Params.SetIntegerParam(DmConsts.EndPositon, Params.GetIntegerParam(DmConsts.StartPosition));
            }

            for (int indSearchParam = Params.GetIntegerParam(DmConsts.LastPosition); indSearchParam <= Params.GetIntegerParam(DmConsts.EndPositon); indSearchParam++)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                Params.SetIntegerParam(DmConsts.LastPosition, indSearchParam);
                UpdateProgress();
                WriteLog(String.Format("start processing county  [{0}][{1}]; page [{2}] ",
                       Params.GetIntegerParam(DmConsts.LastPosition),
                       _counties[Params.GetIntegerParam(DmConsts.LastPosition)],
                       Params.GetIntegerParam(DmConsts.LastPage)));

                NavigateByPage(_counties[Params.GetIntegerParam(DmConsts.LastPosition)]);
            }
        }


        private void SearchingByNames()
        {
            if (Test)
            {
                Params.SetIntegerParam(DmConsts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
                Params.SetIntegerParam(DmConsts.EndPositon, Params.GetIntegerParam(DmConsts.StartPosition));
            }

            FileInfo fi = new FileInfo(String.Format("{0}\\{1}", SettingsDir, SearchParamFileName));
            if (!fi.Exists)
                throw new Exception("File not found: " + fi.FullName);

            int iSearchParam;
            using (StreamReader sr = fi.OpenText())
            {
                iSearchParam = 0;
                while (sr.ReadLine() != null)
                {
                    if (Ct.IsCancellationRequested) break;
                    iSearchParam++;
                }
            }

            if (Params.GetIntegerParam(DmConsts.EndPositon) > iSearchParam)
                Params.SetIntegerParam(DmConsts.EndPositon, iSearchParam);

            iSearchParam = 0;
            using (StreamReader sr = fi.OpenText())
            {
                string sSearchParam;
                while ((sSearchParam = sr.ReadLine()) != null) //loop by search params
                {
                    if (Ct.IsCancellationRequested) break;
                    if (iSearchParam > Params.GetIntegerParam(DmConsts.EndPositon)) break;

                    if (iSearchParam >= Params.GetIntegerParam(DmConsts.LastPosition) && !string.IsNullOrEmpty(sSearchParam))
                    {
                        Params.SetIntegerParam(DmConsts.LastPosition, iSearchParam);
                        UpdateProgress();
                        HtmlUtils.PrepareSearchString(ref sSearchParam);

                        WriteLog(String.Format("start processing lastname  [{0}][{1}]; page [{2}] ",
                            Params.GetIntegerParam(DmConsts.LastPosition),
                            sSearchParam,
                            Params.GetIntegerParam(DmConsts.LastPage)));
                        NavigateByPage(sSearchParam);
                    } //if (iname >= FLastPositionParam)  
                    iSearchParam++;
                } //while ((sparam = sr.ReadLine()) != null) //loop by search params
            } //using (StreamReader sr = fi.OpenText())
        }

        private void NavigateByPage(string searchparam)
        {
            string count = "";
            string searchUrl = (Params.GetBooleanParam(DownloadByCounties)) ? _searchUrlByCounties : _searchUrlByNames;
            do
            {
                if (Ct.IsCancellationRequested)
                    break;

                WriteLog(String.Format("Navigate to page [{0}] ",
                   Params.GetIntegerParam(DmConsts.LastPage)));

                string html = GetRequest.UrlOnly(
                    string.Format(searchUrl, searchparam, Params.GetIntegerParam(DmConsts.LastPage)),
                    Rpm);

                if (html.Contains("Ethnicity:"))
                {
                    string lookupKey = RegExpr.Parse(html, @"(?si:searchby=publicinput&id=([^""]+)"")").Trim();
                    if (string.IsNullOrEmpty(lookupKey))
                        throw new Exception(String.Format("Invalid lookupKey. {0}{1}{2}", DmConsts.ExcNeedRestart, Environment.NewLine, html));

                    WriteLog(String.Format("ParseOffender [{0}][{1}]", Params.GetIntegerParam(DmConsts.LastRecord), lookupKey));
                    ParseOffender(html, lookupKey);
                    UpdateRecordCount(1);
                    break;
                }


                if (string.IsNullOrEmpty(count))
                {
                    count = RegExpr.Parse(html, @"(?si:of\s+([\d]+)\s+offenders)").Trim();
                    WriteLog(string.Format("found [{0}] offenders", count));
                }

                if (RegExpr.IsMatch(html, @"(?si:javascript\: OpenDetail)"))
                {
                    ParseResultPage(html);
                    Params.SetIntegerParam(DmConsts.LastPage, Params.GetIntegerParam(DmConsts.LastPage) + 1);
                }
                else if (RegExpr.IsMatch(html, @"(?si:Your search returned no results)"))
                {
                    WriteLog("No results found.");
                    break;
                }
                else if (RegExpr.IsMatch(html, @"(?si:prosoma.dll\?searchby=offender\&id=)"))
                    break;
                else
                    throw new Exception("Result page: " + DmConsts.ExcNeedRestart);
            }
            while (true);

            Params.SetIntegerParam(DmConsts.LastPage, 1);
        }

        private void ParseResultPage(string html)
        {
            html = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(html.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));

            if (!RegExpr.IsMatch(html, @"(?si:agMegan.gif)"))
                throw new Exception(String.Format("Result page: {0}{1}{2}", DmConsts.ExcNeedRestart, Environment.NewLine, html));

            MatchCollection mc = RegExpr.ParseMatchCollection(html,
                @"(?si:<a HREF=""javascript: OpenDetail\x28\x27([\S]+)\x27\x29"" onmouseover=""window.status=\x27Open Detail\x27; return true;"" onmouseout=""window.status=\x27\x27; return true;""><IMG BORDER)");
            WriteLog(String.Format("found [{0}] matches", mc.Count));
            if (mc.Count == 0)
                throw new Exception(DmConsts.ExcNeedRestart + Environment.NewLine + html);

            List<string> list = new List<string>();

            foreach (Match matchLoc in mc)
                if (!list.Contains(matchLoc.Groups[1].Value))
                    list.Add(matchLoc.Groups[1].Value);

            WriteLog(String.Format("found [{0}] id[s]", list.Count));

            for (int i = Params.GetIntegerParam(DmConsts.LastRecord); i < list.Count; i++)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                Params.SetIntegerParam(DmConsts.LastRecord, i);
                string lookupKey = list[i];
                string offenderHtml = GetRequest.WithCookies(string.Format(_offenderLink, lookupKey), Rpm);
                WriteLog(String.Format("ParseOffender [{0}][{1}]", Params.GetIntegerParam(DmConsts.LastRecord), lookupKey));

                if (!RegExpr.IsMatch(offenderHtml, @"(?si:Last Name)"))
                    throw new Exception("Offenders page:" + DmConsts.ExcNeedRestart);

                ParseOffender(offenderHtml, lookupKey);
                UpdateRecordCount(1);
            }
            Params.SetIntegerParam(DmConsts.LastRecord, 0);
        }

        private void ParseOffender(string offenderHtml, string lookupKey)
        {
            offenderHtml = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(offenderHtml.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));
            using (OffendersResult or = new OffendersResult())
            {
                or.Source_State = StateName;
                or.Source_id = lookupKey;
                or.offenderLink = string.Format(_offenderLink, lookupKey);

                or.firstname = RegExpr.Parse(offenderHtml, @"(?si:First Name\s*:\s*</font>\s*</font>\s*<font[^<>]*?>\s*<b>(.*?)</b>)").Trim();
                or.lastname = RegExpr.Parse(offenderHtml, @"(?si:Last Name\s*:\s*</font>.*?<b>([A-Z^<>]+)</b>.*?</td>)").Trim();
                if (string.IsNullOrEmpty(or.lastname))
                    throw new Exception("Empty offenders lastname, possible invalid loading page: " + or.offenderLink);

                foreach (Match m in RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, "(?si:Middle Name:</font>(.*?)</td>)")
                   , "(?si:<B>(.*?)</B>)"))
                    if (!string.IsNullOrEmpty(m.Groups[1].Value.Trim()))
                        if (string.IsNullOrEmpty(or.middlename))
                            or.middlename = m.Groups[1].Value.Trim();
                        else or.middlename += "@" + m.Groups[1].Value.Trim();

                foreach (Match matchLoc in RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, @"(?si:<table border=""0""\s*cellpadding=""2""\s*cellspacing=""0""\s*id=""ALI""(.*?)</table>)")
                    , "(?si:<li>(.*?)</li>)"))
                    if (matchLoc.Groups[1].Value.Trim().ToLower() != "none")
                        or.AKA += matchLoc.Groups[1].Value.Trim() + ";";

                or.Address = RegExpr.Parse(offenderHtml, @"(?si:Last Known Address[^<>]*?</td>\s*<td>\s*<b>(.*?)</b>)").Replace("<br>", "@").Trim();
                if (or.Address == "@" || or.Address == "@,") or.Address = "";

                or.County = RegExpr.Parse(offenderHtml, @"(?si:County\s*:\s*([^<>]*?)<)").Trim();
                or.Birthdate = RegExpr.Parse(offenderHtml, @"(?si:Date of Birth\s*:\s*</td>\s*<td[^<>]*?>\s*<B>(.*?)</B>)").Trim();

                or.Ethnicity = RegExpr.Parse(offenderHtml, @"(?si:Ethnicity\s*:\s*</td>\s*<td[^<>]*?>\s*<B>(.*?)</B>)").Trim();
                or.eye_color = RegExpr.Parse(offenderHtml, @"(?si:Eye Color\s*:\s*</td>\s*<td[^<>]*?>\s*<B>(.*?)</B>)").Trim();
                or.hair_color = RegExpr.Parse(offenderHtml, @"(?si:Hair Color\s*:\s*</td>\s*<td[^<>]*?>\s*<B>(.*?)</B>)").Trim();
                or.height = RegExpr.Parse(offenderHtml, @"(?si:Height\s*:\s*</td>\s*<td[^<>]*?>\s*<B>(.*?)</B>)").Trim();
                or.weight = RegExpr.Parse(offenderHtml, @"(?si:Weight\s*:\s*</td>\s*<td[^<>]*?>\s*<B>(.*?)</B>)").Trim();
                or.sex = RegExpr.Parse(offenderHtml, @"(?si:Sex\s*:\s*</td>\s*<td[^<>]*?>\s*<B>(.*?)</B>)").Trim();

                const string crimeCaption = "Offense Code;Description~";
                or.Crime += crimeCaption;
                foreach (Match matchLoc in RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, @"(?si:<table cellpadding=""4""\s*cellspacing=""0""\s*border=""0""\s*id=""OFF""(.*?)</table>)")
                    , @"(?si:<tr valign=""top"">(.*?)</tr>)"))
                {
                    or.Crime += RegExpr.ParseMatchCollection(matchLoc.Groups[1].Value, "(?si:>(.*?)</td>)")[0].Groups[1].Value.Trim() + ";";
                    or.Crime += RegExpr.ParseMatchCollection(matchLoc.Groups[1].Value, "(?si:>(.*?)</td>)")[1].Groups[1].Value.Trim() + "~";
                    if (or.Crime.Contains("<b>"))
                        or.Crime = crimeCaption;
                }

                foreach (Match matchLoc in RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, @"(?si:<table border=""0""\s*cellpadding=""2""\s*cellspacing=""0""\s*id=""SMT""(.*?)</table>)")
                    , "(?si:<li>(.*?)</li>)"))
                    if (matchLoc.Groups[1].Value.Trim().ToLower() != "none")
                        or.marks += matchLoc.Groups[1].Value.Trim() + ";";

                if (RegExpr.IsMatch(offenderHtml, "(?si:THIS SEX OFFENDER HAS BEEN IN VIOLATION)"))
                    or.Status = "IN VIOLATION";

                if (RegExpr.IsMatch(offenderHtml, "(?si:>TRANSIENT<)"))
                    if (string.IsNullOrEmpty(or.Status))
                        or.Status = "TRANSIENT";
                    else or.Status += "; TRANSIENT";

                if (RegExpr.IsMatch(offenderHtml, "(?si:>INCARCERATED<)"))
                    if (string.IsNullOrEmpty(or.Status))
                        or.Status = "INCARCERATED";
                    else or.Status += "; INCARCERATED";

                if (RegExpr.IsMatch(offenderHtml, "(?si:>LOCATION UNKNOWN<)"))
                    if (string.IsNullOrEmpty(or.Status))
                        or.Status = "LOCATION UNKNOWN";
                    else or.Status += "; LOCATION UNKNOWN";

                string imagehtmlUrl = RegExpr.Parse(offenderHtml, String.Format("(?si:http://(.*?)/maps/{0}.jpg)", lookupKey), 0);
                if (Params.GetBooleanParam(DmConsts.IsDownloadImg) && imagehtmlUrl != "")
                {
                    or.Image_ID = Guid.NewGuid() + ".jpg";
                    or.ImageLink = imagehtmlUrl;

                    Offenders.DownloadImageWithExceptions(_imagesResultFile,
                        String.Format("{0}\\{1}\\{2}", ResultDir, DmConsts.PhotosDir, or.Image_ID),
                        or.ImageLink, TaskParams.SessionId);
                }
                if (or.Image_ID != "" && !File.Exists(String.Format("{0}\\{1}\\{2}", ResultDir, DmConsts.PhotosDir, or.Image_ID)))
                    throw new Exception(DmConsts.ExcImgNotFound);

                ResultFile.Write(or.GetValues());
            }
        }

        private void UpdateProgress()
        {
            if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                    DmConsts.StartPosition, DmConsts.EndPositon));
            if (Params.GetIntegerParam(DmConsts.LastPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.LastPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                    DmConsts.StartPosition, DmConsts.EndPositon));
            if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.LastPosition))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.LastPosition),
                    DmConsts.StartPosition, DmConsts.LastPosition));
            UpdateProgress(((double)(Params.GetIntegerParam(DmConsts.LastPosition) - Params.GetIntegerParam(DmConsts.StartPosition))) / (Params.GetIntegerParam(DmConsts.EndPositon) - Params.GetIntegerParam(DmConsts.StartPosition) + 1));
        }
    }
}
